import urllib.robotparser

rp = urllib.robotparser.RobotFileParser()

rp.set_url('http://fanyi.youdao.com/robots.txt')
rp.read()
user_agrent = 'Baiduspider'
url = 'http://fanyi.youdao.com/web2/index.html'
print(rp.can_fetch(user_agrent,url))

print(rp.can_fetch(user_agrent,'https://f.youdao.com/?path=fast&keyfrom=Nav-fast'))